package com.yc.testupload.controller;

import com.yc.testupload.crawler.config.WebsiteConfigManager;
import com.yc.testupload.crawler.task.CrawlTask;
import com.yc.testupload.crawler.task.TaskList;
import com.yc.testupload.crawler.strategy.StrategySelector;
import com.yc.testupload.util.DatabaseUtil;
import jakarta.servlet.ServletException;
import jakarta.servlet.annotation.WebServlet;
import jakarta.servlet.http.HttpServlet;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.servlet.http.HttpSession;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Enumeration;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;

@WebServlet(name = "crawlerServlet", value = "/api/crawl")
public class CrawlerServlet extends HttpServlet {

    // 静态初始化块，记录Servlet类加载信息
    static {
        System.out.println("[静态初始化] CrawlerServlet类正在加载...");
        Logger initLogger = LogManager.getLogger(CrawlerServlet.class);
        initLogger.info("[静态初始化] CrawlerServlet类加载完成");
    }

    // 日志记录器
    private static final Logger logger = LogManager.getLogger(CrawlerServlet.class);
    // 策略选择器
    private StrategySelector strategySelector;
    // 任务列表
    private TaskList taskList;
    // 网站配置管理器
    private WebsiteConfigManager websiteConfigManager;
    // URL变量
    private String url;

    public CrawlerServlet() {
        super();
        System.out.println("[实例化] CrawlerServlet实例被创建");
        logger.info("[实例化] CrawlerServlet实例被创建");
    }

    @Override
    public void init() throws ServletException {
        super.init();
        // 初始化策略选择器
        strategySelector = new StrategySelector();
        // 初始化任务列表
        taskList = TaskList.getInstance();
        // 初始化网站配置管理器
        websiteConfigManager = WebsiteConfigManager.getInstance();
        logger.info("策略选择器、任务列表和网站配置管理器已初始化");
    }

    @Override
    protected void doGet(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        // 立即记录请求到达的基本信息
        System.out.println("[请求接收] doGet方法被调用");
        logger.info("[请求入口] 收到HTTP GET请求: {}", request.getRequestURI());

        // 处理GET请求，调用统一处理方法
        handleCrawlRequest(request, response);
    }

    /**
     * 处理抓取请求
     */
    private void handleCrawlRequest(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        // 设置响应类型为JSON
        response.setContentType("application/json");
        response.setCharacterEncoding("UTF-8");

        // 获取请求参数
        String requestUrl = null; // 本地变量，用于临时存储请求中的URL
        String site = null;

        // 检查Content-Type是否为JSON
        String contentType = request.getContentType();
        String jsonBody = null;
        if (contentType != null && contentType.contains("application/json")) {
            // 从JSON请求体中解析参数
            StringBuilder sb = new StringBuilder();
            try (BufferedReader reader = request.getReader()) {
                String line;
                while ((line = reader.readLine()) != null) {
                    sb.append(line);
                }
            }

            jsonBody = sb.toString();
            logger.info("收到JSON请求体: {}", jsonBody);

            // 简单的JSON解析
            if (jsonBody.contains("\"url\":")) {
                int urlStart = jsonBody.indexOf("\"url\":") + 7;
                int urlEnd = jsonBody.indexOf(",", urlStart);
                if (urlEnd == -1)
                    urlEnd = jsonBody.indexOf("}", urlStart);
                requestUrl = jsonBody.substring(urlStart, urlEnd).replace("'", "").replace("\"", "").trim();
            }

            if (jsonBody.contains("\"site\":")) {
                int siteStart = jsonBody.indexOf("\"site\":") + 8;
                int siteEnd = jsonBody.indexOf(",", siteStart);
                if (siteEnd == -1)
                    siteEnd = jsonBody.indexOf("}", siteStart);
                site = jsonBody.substring(siteStart, siteEnd).replace("'", "").replace("\"", "").trim();
            }
        } else {
            // 回退到传统方式获取参数
            requestUrl = request.getParameter("url");
            site = request.getParameter("site");
        }

        // 设置全局URL变量
        this.url = requestUrl != null ? requestUrl : "";

        logger.info("获取到请求URL: {}, 网站: {}", this.url, site);

        // 验证URL参数
        if (this.url == null || this.url.trim().isEmpty()) {
            String errorResponse = "{\"success\": false, \"message\": \"URL参数不能为空\"}";
            response.getWriter().write(errorResponse);
            logger.warn("请求URL为空");
            return;
        }

        try {
            // 从URL获取内容
            String content = fetchContentFromUrl(this.url);
            if (content == null || content.isEmpty()) {
                String errorResponse = "{\"success\": false, \"message\": \"获取URL内容失败\"}";
                response.getWriter().write(errorResponse);
                logger.warn("获取URL内容为空");
                return;
            }

            // 使用策略选择器提取问题
            String questions = strategySelector.extractQuestions(this.url, content);
            if (questions == null || questions.isEmpty()) {
                // 如果策略选择器未找到匹配策略，回退到原有提取方法
                logger.info("策略选择器未找到匹配策略，使用回退方法提取");
                questions = extractInterviewQuestions(content);
            }

            // 提取标题
            String title = extractTitleFromHtml(content);

            // 检查是否需要保存到数据库（默认为false）
            boolean shouldSaveToDb = false;
            if (contentType != null && contentType.contains("application/json")) {
                // 从JSON请求体中检查是否需要保存到数据库
                shouldSaveToDb = jsonBody.contains("\"saveToDb\":true");
            } else {
                // 从请求参数中检查是否需要保存到数据库
                String saveParam = request.getParameter("saveToDb");
                shouldSaveToDb = "true".equals(saveParam);
            }

            int docId = -1;
            // 构建JSON响应
            String jsonResponse = "{\"success\": true, \"docId\": " + docId + ", \"title\": \"" + escapeJson(title)
                    + "\", \"questions\": \"" + escapeJson(questions) + "\"}";
            response.getWriter().write(jsonResponse);

            logger.info("请求处理成功，文档ID: {}", docId);
        } catch (Exception e) {
            // 处理异常
            logger.error("处理请求时出错: {}", e.getMessage());
            String errorResponse = "{\"success\": false, \"message\": \"处理请求时出错: " + escapeJson(e.getMessage()) + "\"}";
            response.getWriter().write(errorResponse);
        }
    }

    /**
     * 处理保存到素材表请求
     */
    private void handleSaveToMaterial(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        // 设置响应类型为JSON
        response.setContentType("application/json");
        response.setCharacterEncoding("UTF-8");

        try {
            // 获取请求参数
            String title = request.getParameter("title");
            String content = request.getParameter("content");
            String url = request.getParameter("url");
            String categoryId = request.getParameter("categoryId");

            // 从会话中获取用户ID
            HttpSession session = request.getSession(false);
            Integer userIdObj = null;
            if (session != null) {
                userIdObj = (Integer) session.getAttribute("userId");
            }

            // 验证必要参数
            if (title == null || title.trim().isEmpty() || content == null || content.trim().isEmpty()) {
                String errorResponse = "{\"success\": false, \"message\": \"标题和内容不能为空\"}";
                response.getWriter().write(errorResponse);
                logger.warn("保存到素材表失败: 标题或内容为空");
                return;
            }

            // 验证用户是否已登录
            if (userIdObj == null) {
                String errorResponse = "{\"success\": false, \"message\": \"请先登录后再保存素材\"}";
                response.getWriter().write(errorResponse);
                logger.warn("保存到素材表失败: 用户未登录");
                return;
            }

            // 设置默认值
            if (categoryId == null || categoryId.trim().isEmpty()) {
                categoryId = "31"; // 默认分类ID
            }

            // 将Integer类型的userId转换为String类型
            String userId = userIdObj.toString();

            // 保存到素材表
            int materialId = saveToMaterialTable(title, content, categoryId, userId, url);

            // 构建JSON响应
            String jsonResponse = "{\"success\": true, \"materialId\": " + materialId + ", \"message\": \"保存成功\"}";
            response.getWriter().write(jsonResponse);

            logger.info("保存到素材表成功，素材ID: {}", materialId);
        } catch (Exception e) {
            // 处理异常
            logger.error("保存到素材表时出错: {}", e.getMessage());
            String errorResponse = "{\"success\": false, \"message\": \"保存失败: " + escapeJson(e.getMessage()) + "\"}";
            response.getWriter().write(errorResponse);
        }
    }

    /**
     * 保存到素材表
     */
    private int saveToMaterialTable(String title, String content, String categoryId, String userId, String url) {
        logger.info("开始将内容保存到素材表");
        long startTime = System.currentTimeMillis();
        int materialId = -1;
        Connection conn = null;
        PreparedStatement pstmt = null;
        ResultSet rs = null;

        try {
            // 获取数据库连接
            conn = DatabaseUtil.getConnection();
            logger.info("已获取数据库连接");

            // 获取当前时间
            String currentTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date());

            // 保存到wk_material表
            String sql = "INSERT INTO wk_material (title, content, category_id, user_id, source_url, add_time, edit_time, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
            pstmt = conn.prepareStatement(sql, PreparedStatement.RETURN_GENERATED_KEYS);
            pstmt.setString(1, title);
            pstmt.setString(2, content);
            pstmt.setInt(3, Integer.parseInt(categoryId));
            pstmt.setInt(4, Integer.parseInt(userId));
            pstmt.setString(5, url != null ? url : "");
            pstmt.setString(6, currentTime);
            pstmt.setString(7, currentTime);
            pstmt.setInt(8, 1); // 状态设为1（可用）

            int rowsAffected = pstmt.executeUpdate();
            logger.info("执行SQL语句影响的行数: {}", rowsAffected);

            // 获取生成的素材ID
            rs = pstmt.getGeneratedKeys();
            if (rs.next()) {
                materialId = rs.getInt(1);
                logger.info("成功保存到素材表，素材ID: {}", materialId);
            }

        } catch (SQLException e) {
            logger.error("保存到素材表失败: {}", e.getMessage(), e);
        } finally {
            // 关闭数据库资源
            if (rs != null) {
                try {
                    rs.close();
                } catch (SQLException e) {
                    logger.error("关闭ResultSet失败", e);
                }
            }
            if (pstmt != null) {
                try {
                    pstmt.close();
                } catch (SQLException e) {
                    logger.error("关闭PreparedStatement失败", e);
                }
            }
            DatabaseUtil.closeConnection(conn);
        }

        long endTime = System.currentTimeMillis();
        logger.info("素材表操作完成，耗时: {} ms", (endTime - startTime));
        return materialId;
    }

    /**
     * 从HTML中提取标题
     */
    private String extractTitleFromHtml(String html) {
        String title = "无标题文档";
        try {
            Pattern titlePattern = Pattern.compile("<title>(.*?)</title>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
            Matcher titleMatcher = titlePattern.matcher(html);
            if (titleMatcher.find()) {
                title = titleMatcher.group(1).trim();
            }
        } catch (Exception e) {
            logger.error("提取标题时出错: {}", e.getMessage());
        }
        return title;
    }

    /**
     * 从指定URL获取内容
     */
    // 信任所有SSL证书的TrustManager实现
    private static class TrustAllManager implements TrustManager, javax.net.ssl.X509TrustManager {
        @Override
        public void checkClientTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) {}

        @Override
        public void checkServerTrusted(java.security.cert.X509Certificate[] x509Certificates, String s) {}

        @Override
        public java.security.cert.X509Certificate[] getAcceptedIssuers() {
            return new java.security.cert.X509Certificate[0];
        }
    }

    // 初始化SSL上下文，信任所有证书和主机名
    private void initSSLContext() {
        try {
            SSLContext sslContext = SSLContext.getInstance("TLS");
            sslContext.init(null, new TrustManager[]{new TrustAllManager()}, new java.security.SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
            HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);
            logger.info("SSL上下文初始化成功，已设置信任所有证书");
        } catch (Exception e) {
            logger.error("初始化SSL上下文失败: {}", e.getMessage());
        }
    }

    private String fetchContentFromUrl(String urlStr) throws IOException {
        logger.info("开始抓取URL: {}", urlStr);
        long startTime = System.currentTimeMillis();

        // 初始化SSL上下文，信任所有证书
        initSSLContext();

        URL url = new URL(urlStr);
        HttpURLConnection conn = null;
        try {
            // 尝试直接连接，如果失败则尝试使用代理
            try {
                conn = (HttpURLConnection) url.openConnection();
                conn.setConnectTimeout(5000); // 先设置较短的超时时间测试直接连接
                conn.connect();
                conn.disconnect(); // 关闭测试连接
                logger.info("直接连接成功，继续使用直接连接");
            } catch (IOException e) {
                logger.warn("直接连接失败: {}, 尝试使用系统默认代理设置", e.getMessage());
                // 使用系统默认代理
                System.setProperty("java.net.useSystemProxies", "true");
            }

            // 重新创建连接
            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setConnectTimeout(20000); // 增加连接超时时间到20秒
            conn.setReadTimeout(30000); // 增加读取超时时间到30秒
            conn.setRequestProperty("User-Agent",
                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
            conn.setRequestProperty("Accept",
                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
            conn.setRequestProperty("Cookie", "NOWCODERCLINETID=1234567890ABCDEF; SERVERID=123456"); // 添加Cookie以模拟登录状态
            conn.setInstanceFollowRedirects(true); // 自动跟随重定向

            int responseCode = conn.getResponseCode();
            logger.info("URL响应码: {}", responseCode);

            if (responseCode != HttpURLConnection.HTTP_OK) {
                throw new IOException("服务器返回非成功状态码: " + responseCode + " (" + urlStr + ")");
            }

            // 获取响应
            StringBuilder response = new StringBuilder();
            try (BufferedReader reader = new BufferedReader(
                    new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
                String line;
                while ((line = reader.readLine()) != null) {
                    response.append(line).append("\n");
                }
            }

            // 保存原始HTML内容到文件，用于分析
            saveHtmlToFile(response.toString());

            long endTime = System.currentTimeMillis();
            logger.info("URL抓取完成，耗时: {} ms, 内容长度: {} 字符",
                    (endTime - startTime), response.length());

            return response.toString();
        } catch (IOException e) {
            logger.error("抓取URL内容失败: {}", e.getMessage());
            throw e;
        } finally {
            if (conn != null) {
                conn.disconnect();
            }
        }
    }

    /**
     * 保存HTML内容到文件，用于调试和分析
     */
    private void saveHtmlToFile(String html) {
        try {
            // 创建保存目录
            String dirPath = "e:/develop/testUpload/saved_html";
            File dir = new File(dirPath);
            if (!dir.exists()) {
                dir.mkdirs();
            }

            // 生成唯一的文件名
            String fileName = "html_" + System.currentTimeMillis() + ".html";
            String filePath = dirPath + File.separator + fileName;

            // 保存内容到文件
            try (PrintWriter writer = new PrintWriter(new FileWriter(filePath, StandardCharsets.UTF_8))) {
                writer.write(html);
            }

            logger.info("HTML内容已保存到: {}", filePath);
        } catch (IOException e) {
            logger.error("保存HTML内容到文件失败: {}", e.getMessage());
        }
    }

    /**
     * 提取面试题
     */
    private String extractInterviewQuestions(String html) {
        // 首先尝试使用专用的牛客网提取方法
        String niukeResult = extractNiuKeInterviewQuestions(html);
        if (!niukeResult.isEmpty()) {
            return niukeResult;
        }

        // 然后尝试使用通用题目-答案格式提取
        String commonResult = extractCommonQuestionAnswerFormat(html);
        if (!commonResult.isEmpty()) {
            return commonResult;
        }

        // 最后尝试从JavaScript状态对象中提取
        String jsStateResult = extractInterviewQuestionsFromJsState(html);
        if (!jsStateResult.isEmpty()) {
            return jsStateResult;
        }

        // 如果所有方法都失败，返回一个友好的提示
        return "未能提取到有效的面试题内容，请确保URL指向正确的面试题页面，可能需要提供Cookie信息。";
    }

    /**
     * 从JavaScript状态对象中提取面试题
     */
    private String extractInterviewQuestionsFromJsState(String html) {
        StringBuilder result = new StringBuilder();
        try {
            // 尝试匹配JavaScript状态对象中的题目数据
            Pattern pattern = Pattern.compile("window\\.INITIAL_STATE\\s*=\\s*(\\{.*?\\});", Pattern.DOTALL);
            Matcher matcher = pattern.matcher(html);
            
            if (matcher.find()) {
                String jsonData = matcher.group(1);
                // 简单提取JSON中的文本内容，实际应用中应该使用JSON解析库
                result.append("从JavaScript状态对象提取到的数据:\n");
                // 这里只是简单示例，实际应该使用Jackson或Gson等库解析JSON
                result.append(jsonData.substring(0, Math.min(500, jsonData.length())));
            }
        } catch (Exception e) {
            logger.error("从JavaScript状态对象提取面试题失败: {}", e.getMessage());
        }
        return result.toString();
    }

    /**
     * 提取牛客网面试题格式
     */
    private String extractNiuKeInterviewQuestions(String html) {
        StringBuilder result = new StringBuilder();

        try {
            // 优化正则表达式，避免贪婪匹配导致的性能问题
            // 先移除HTML标签，减少处理的复杂度
            String text = html.replaceAll("<[^>]*>", "").replaceAll("\\s+", " ").trim();

            // 分段处理文本，避免一次性处理过大的文本
            String[] paragraphs = text.split("\\n+");
            boolean inQuestion = false;
            StringBuilder currentQuestion = new StringBuilder();
            StringBuilder currentAnswer = new StringBuilder();
            int count = 0;

            for (String paragraph : paragraphs) {
                paragraph = paragraph.trim();
                if (paragraph.isEmpty())
                    continue;

                // 检查是否是题目开始
                if ((paragraph.matches("^\\d+\\..*$") ||
                        paragraph.matches("^\\d+、.*$") ||
                        paragraph.contains("问题") ||
                        paragraph.contains("题目")) &&
                        count < 30) { // 增加题目数量上限

                    // 如果之前有未完成的题目，先保存
                    if (inQuestion && currentQuestion.length() > 0) {
                        result.append("题目 " + count + ": " + currentQuestion.toString().trim()).append("\n");
                        if (currentAnswer.length() > 0) {
                            result.append("答案: " + currentAnswer.toString().trim()).append("\n\n");
                            currentAnswer.setLength(0);
                        }
                    }

                    // 开始新题目
                    currentQuestion.setLength(0);
                    currentQuestion.append(paragraph);
                    inQuestion = true;
                    count++;
                }
                // 检查是否是答案开始
                else if (inQuestion && (paragraph.contains("答案") || paragraph.contains("解析"))) {
                    currentAnswer.append(paragraph);
                }
                // 属于当前问题或答案的内容
                else if (inQuestion) {
                    if (paragraph.contains("答案") || paragraph.contains("解析")) {
                        currentAnswer.append(paragraph);
                    } else {
                        currentQuestion.append(" " + paragraph);
                    }
                }
            }

            // 保存最后一个题目
            if (inQuestion && currentQuestion.length() > 0 && count > 0) {
                result.append("题目 " + count + ": " + currentQuestion.toString().trim()).append("\n");
                if (currentAnswer.length() > 0) {
                    result.append("答案: " + currentAnswer.toString().trim()).append("\n\n");
                }
            }
        } catch (Exception e) {
            logger.error("提取牛客网面试题格式时出错: {}", e.getMessage());
        }

        return result.toString();
    }

    /**
     * 提取通用的题目-答案格式
     */
    private String extractCommonQuestionAnswerFormat(String html) {
        StringBuilder result = new StringBuilder();

        try {
            // 优化正则表达式，避免复杂的先行断言导致的性能问题
            // 先移除HTML标签，减少处理的复杂度
            String text = html.replaceAll("<[^>]*>", "").replaceAll("\\s+", " ").trim();

            // 使用更简单的方式提取数字序号的题目
            Pattern numberedPattern = Pattern.compile("(\\d+\\.|\\d+、)(.*?)", Pattern.CASE_INSENSITIVE);
            Matcher numberedMatcher = numberedPattern.matcher(text);

            int count = 0;
            int lastEnd = 0;
            while (numberedMatcher.find(lastEnd) && count < 10) {
                int start = numberedMatcher.start();
                int end = numberedMatcher.end();

                // 提取题目内容（包括序号）
                String question = numberedMatcher.group(0).trim();

                // 尝试查找答案部分
                int answerStart = text.indexOf("答案", end);
                int answerStart2 = text.indexOf("解析", end);
                int nextQuestionStart = -1;

                // 查找下一个题目的开始位置
                Matcher nextMatcher = Pattern.compile("(\\d+\\.|\\d+、)", Pattern.CASE_INSENSITIVE).matcher(text);
                if (nextMatcher.find(end)) {
                    nextQuestionStart = nextMatcher.start();
                }

                // 提取答案（如果有）
                if (answerStart > 0 && (nextQuestionStart == -1 || answerStart < nextQuestionStart)) {
                    String answer;
                    if (nextQuestionStart > 0) {
                        answer = text.substring(answerStart, nextQuestionStart).trim();
                        lastEnd = nextQuestionStart;
                    } else {
                        answer = text.substring(answerStart).trim();
                        lastEnd = text.length();
                    }
                    result.append("题目 " + (count + 1) + ": " + question).append("\n");
                    result.append(answer).append("\n\n");
                } else if (answerStart2 > 0 && (nextQuestionStart == -1 || answerStart2 < nextQuestionStart)) {
                    String answer;
                    if (nextQuestionStart > 0) {
                        answer = text.substring(answerStart2, nextQuestionStart).trim();
                        lastEnd = nextQuestionStart;
                    } else {
                        answer = text.substring(answerStart2).trim();
                        lastEnd = text.length();
                    }
                    result.append("题目 " + (count + 1) + ": " + question).append("\n");
                    result.append(answer).append("\n\n");
                } else {
                    // 如果没有明确的答案部分，只提取题目
                    result.append("题目 " + (count + 1) + ": " + question).append("\n\n");
                    lastEnd = end;
                }

                count++;
            }
        } catch (Exception e) {
            logger.error("提取通用题目-答案格式时出错: {}", e.getMessage());
        }

        return result.toString();
    }

    /**
     * 转义JSON字符串中的特殊字符
     */
    private String escapeJson(String text) {
        if (text == null)
            return "";
        return text
                .replace("\\", "\\\\")
                .replace("\"", "\\\"")
                .replace("\b", "\\b")
                .replace("\f", "\\f")
                .replace("\n", "\\n")
                .replace("\r", "\\r")
                .replace("\t", "\\t");
    }

    /**
     * 根据URL获取对应的分类ID
     * 
     * @param url 要抓取的URL
     * @return 分类ID
     */
    private int getCategoryIdForUrl(String url) {
        // 首先尝试从任务列表中获取分类ID
        CrawlTask task = taskList.getTaskByUrl(url);
        if (task != null) {
            logger.info("通过任务列表获取到分类ID: {}，对应网站: {}, 频道: {}",
                    task.getCategoryId(), task.getWebsiteName(), task.getChannelName());
            return task.getCategoryId();
        }

        // 如果没有匹配的任务，默认使用面试题分类ID（31）
        logger.warn("未找到与URL: {} 匹配的任务，使用默认分类ID: 31（面试题）", url);
        return 31;
    }

    @Override
    public void destroy() {
        // 清理资源
        logger.info("爬虫Servlet已销毁");
        super.destroy();
    }

    /**
     * 测试方法：直接从本地HTML文件提取面试题，用于调试
     */
    public static void main(String[] args) {
        // 初始化Logger
        org.apache.logging.log4j.Logger logger = org.apache.logging.log4j.LogManager.getLogger(CrawlerServlet.class);
        logger.info("开始测试面试题提取逻辑...");

        // 读取本地HTML文件路径
        String filePath = "e:/develop/testUpload/saved_html/html_1760018093627.html";
        logger.info("读取文件: {}", filePath);

        try {
            // 读取文件内容
            String htmlContent = new String(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(filePath)),
                    "UTF-8");
            logger.info("文件读取成功，长度: {} 字符", htmlContent.length());

            // 创建CrawlerServlet实例
            CrawlerServlet servlet = new CrawlerServlet();

            // 直接调用extractInterviewQuestions方法测试提取逻辑
            String extractedQuestions = servlet.extractInterviewQuestions(htmlContent);

            // 输出提取结果
            logger.info("面试题提取结果长度: {} 字符", extractedQuestions.length());
            logger.info("面试题提取结果预览:\n{}", extractedQuestions.substring(0, Math.min(500, extractedQuestions.length())));

            // 如果提取结果为空，尝试直接调用extractInterviewQuestionsFromJsState方法
            if (extractedQuestions.isEmpty()) {
                logger.info("extractInterviewQuestions结果为空，尝试直接调用extractInterviewQuestionsFromJsState...");
                extractedQuestions = servlet.extractInterviewQuestionsFromJsState(htmlContent);
                logger.info("extractInterviewQuestionsFromJsState结果长度: {} 字符", extractedQuestions.length());
                logger.info("extractInterviewQuestionsFromJsState结果预览:\n{}",
                        extractedQuestions.substring(0, Math.min(500, extractedQuestions.length())));
            }

        } catch (Exception e) {
            logger.error("测试过程中发生错误: {}", e.getMessage());
            e.printStackTrace();
        }

        logger.info("测试完成");
    }

    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        response.setContentType("application/json");
        response.setCharacterEncoding("UTF-8");

        String pathInfo = request.getPathInfo();
        if (pathInfo == null) {
            pathInfo = "";
        }

        if ("/saveToMaterial".equals(pathInfo)) {
            handleSaveToMaterial(request, response);
        } else {
            handleCrawlRequest(request, response);
        }
    }
}